# -*- coding: utf-8 -*-

import datetime

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

from scrapy.http import Headers

from ArticleSpider.items import LagouItemLoader,LagouJobItem
from ArticleSpider.util.common import get_md5

class LagouSpider(CrawlSpider):
    name = 'lagou'
    allowed_domains = ['www.lagou.com']
    start_urls = ['https://www.lagou.com/']

    headers = {
        # "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.30"
    }

    rules = (
        #这样的界面有职位url
        #含有招聘的页面，点击分类后
        #https://www.lagou.com/zhaopin/Java/?labelWords=label
        Rule(LinkExtractor(allow=r'zhaopin/.*'), follow=True,process_request="process_request"),

        #公司详情
        #https://www.lagou.com/gongsi/59251.html
        Rule(LinkExtractor(allow=r'gongsi/\d+.html'), follow=True,process_request="process_request"),

        #职位
        #https://www.lagou.com/jobs/5364501.html
        Rule(LinkExtractor(allow=r'jobs/\d+.html'), callback='parse_item', follow=True,process_request="process_request"),
    )

    def process_request(self,request):
        request.headers = Headers(self.headers)
        # print("process_request")

        return request

    def parse_item(self, response):
        # i = {}
        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
        #i['name'] = response.xpath('//div[@id="name"]').extract()
        #i['description'] = response.xpath('//div[@id="description"]').extract()

        item_loader=LagouItemLoader(item=LagouJobItem(),response=response)
        item_loader.add_value("url",response.url)
        item_loader.add_value("url_object_id",get_md5(response.url))
        item_loader.add_css("title", ".job-name::attr(title)")
        item_loader.add_css("salary", ".job_request .salary::text")

        item_loader.add_xpath("city", "//*[@class='job_request']/p/span[2]/text()")
        item_loader.add_xpath("work_years", "//*[@class='job_request']/p/span[3]/text()")
        item_loader.add_xpath("degree_need", "//*[@class='job_request']/p/span[4]/text()")
        item_loader.add_xpath("type", "//*[@class='job_request']/p/span[5]/text()")

        item_loader.add_css("tags", ".position-label li::text")
        item_loader.add_css("publish_at", ".publish_time::text")
        item_loader.add_css("advantage", ".job-advantage p::text")
        item_loader.add_css("desc", ".job_bt .job-detail")
        item_loader.add_css("address", ".work_addr")
        item_loader.add_css("company_url", "#job_company dt a::attr(href)")
        item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")

        item_loader.add_value("crawl_at", datetime.datetime.now())

        item=item_loader.load_item()

        return item
